{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Phase 3 Weighted Bagging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\gensim\\utils.py:1197: UserWarning: detected Windows; aliasing chunkize to chunkize_serial\n",
      "  warnings.warn(\"detected Windows; aliasing chunkize to chunkize_serial\")\n",
      "Using TensorFlow backend.\n",
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\fuzzywuzzy\\fuzz.py:35: UserWarning: Using slow pure-python SequenceMatcher. Install python-Levenshtein to remove this warning\n",
      "  warnings.warn('Using slow pure-python SequenceMatcher. Install python-Levenshtein to remove this warning')\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "\n",
    "import os\n",
    "import re\n",
    "import csv\n",
    "import codecs\n",
    "import gensim\n",
    "import itertools\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import operator\n",
    "import sys\n",
    "\n",
    "from nltk import ngrams\n",
    "from collections import Counter\n",
    "from string import punctuation\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "from iwillwin.trainer.supervised_trainer import KerasModelTrainer\n",
    "from iwillwin.data_utils.data_helpers import DataTransformer, DataLoader\n",
    "from iwillwin.config import dataset_config\n",
    "from iwillwin.data_utils.feature_engineering import FeatureCreator\n",
    "\n",
    "from fuzzywuzzy import fuzz\n",
    "from nltk.corpus import stopwords\n",
    "from tqdm import tqdm\n",
    "from scipy.stats import skew, kurtosis\n",
    "from scipy.spatial.distance import cosine, cityblock, jaccard, canberra, euclidean, minkowski, braycurtis\n",
    "from nltk import word_tokenize\n",
    "\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "\n",
    "import lightgbm as lgb\n",
    "from sklearn.model_selection import train_test_split\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import KFold\n",
    "\n",
    "import os\n",
    "import re\n",
    "import csv\n",
    "import codecs\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import operator\n",
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "\n",
    "########################################\n",
    "## import packages\n",
    "########################################\n",
    "import os\n",
    "import re\n",
    "import csv\n",
    "import codecs\n",
    "import numpy as np\n",
    "np.random.seed(1337)\n",
    "\n",
    "import pandas as pd\n",
    "import operator\n",
    "import sys\n",
    "\n",
    "from string import punctuation\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "from iwillwin.trainer.supervised_trainer import KerasModelTrainer\n",
    "from iwillwin.data_utils.data_helpers import DataTransformer, DataLoader\n",
    "from iwillwin.config import dataset_config\n",
    "from keras.utils import to_categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Loading model from cache C:\\Users\\zake7\\AppData\\Local\\Temp\\jieba.cache\n",
      "Loading model cost 0.465 seconds.\n",
      "Prefix dict has been built succesfully.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[DataHelper] Apply normalization on value-type columns\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\validation.py:475: DataConversionWarning: Data with input dtype int64 was converted to float64 by MinMaxScaler.\n",
      "  warnings.warn(msg, DataConversionWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Doing preprocessing...\n",
      "Transforming words to indices...\n",
      "Shape of data tensor: (320552, 50) (320552, 50)\n",
      "Shape of label tensor: (320552,)\n",
      "Preprocessed.\n",
      "Number of unique words 83265\n"
     ]
    }
   ],
   "source": [
    "NB_WORDS, MAX_SEQUENCE_LENGTH = 50000, 50\n",
    "data_transformer = DataTransformer(max_num_words=NB_WORDS, max_sequence_length=MAX_SEQUENCE_LENGTH, char_level=False,\n",
    "                                   normalization=True, features_processed=True)\n",
    "trains_nns, tests_nns, labels = data_transformer.prepare_data(dual=False)\n",
    "print(\"Number of unique words\", len(data_transformer.tokenizer.index_docs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "trains_meta = trains_nns[2]\n",
    "tests_meta = tests_nns[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_df = pd.read_csv('../data/dataset/train.csv')\n",
    "test_df = pd.read_csv('../data/dataset/test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nan <class 'float'>\n",
      "nan <class 'float'>\n",
      "nan <class 'float'>\n",
      "nan <class 'float'>\n",
      "nan <class 'float'>\n",
      "nan <class 'float'>\n",
      "nan <class 'float'>\n",
      "nan <class 'float'>\n"
     ]
    }
   ],
   "source": [
    "rumor_words = ['辟谣', '谣言', '勿传', '假的']\n",
    "\n",
    "def is_rumor(text):\n",
    "    if type(text) != str:\n",
    "        print(text, type(text))\n",
    "        return 0\n",
    "    for rumor_word in rumor_words:\n",
    "        if rumor_word in text:\n",
    "            return 1\n",
    "    return 0\n",
    "\n",
    "def has_split_symbol(text):\n",
    "    if type(text) != str:\n",
    "        return 0\n",
    "    if '|' in text:\n",
    "        return 1\n",
    "    return 0\n",
    "\n",
    "for df in [train_df, test_df]:\n",
    "    df['has_|'] = df['title2_zh'].apply(has_split_symbol)\n",
    "    df['has_rumor_words'] = df['title2_zh'].apply(is_rumor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_has_rumor = train_df.has_rumor_words.values\n",
    "test_has_rumor = test_df.has_rumor_words.values\n",
    "\n",
    "trick_trains_features = np.concatenate((trains_nns[2], train_has_rumor.reshape((-1, 1))), axis=1)\n",
    "trick_tests_features = np.concatenate((tests_nns[2], test_has_rumor.reshape((-1, 1))), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "oof_file_names = sorted([f for f in listdir('../data/ensemble/oofs/') if isfile(join('../data/ensemble/oofs/', f)) and f != '.gitkeep'])\n",
    "preds_file_names = [name.replace('-Train', '') for name in oof_file_names]\n",
    "\n",
    "oofs = []\n",
    "preds = []\n",
    "for name in oof_file_names:\n",
    "    oofs.append(pd.read_csv('../data/ensemble/oofs/' + name))\n",
    "for name in preds_file_names:\n",
    "    preds.append(pd.read_csv('../data/ensemble/preds/' + name))    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 AddNN-Ensemble-Train-L0.861201-NB50000.csv\n",
      "1 LightGBM-Ensemble-Train-L0.860117-NB50000.csv\n",
      "2 LightGBMWordLevel-Ensemble-Train-L0.859096-NB50000.csv\n",
      "3 Logistic-Ensemble-Train-L0.860191-NB50000.csv\n",
      "4 Ridge-Ensemble-Train-L0.859993-NB50000.csv\n"
     ]
    }
   ],
   "source": [
    "for i, name in enumerate(oof_file_names):\n",
    "    print(i, name)\n",
    "    \n",
    "trains = pd.DataFrame()\n",
    "tests = pd.DataFrame()\n",
    "\n",
    "for i in range(len(oof_file_names)):\n",
    "    for label_type in ['agreed', 'disagreed', 'unrelated']:\n",
    "        trains['oofs_{}_{}'.format(i, label_type)] = oofs[i][label_type].values\n",
    "        tests['oofs_pred{}_{}'.format(i, label_type)] = preds[i][label_type].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "unrelated = pd.DataFrame()\n",
    "agreeds = pd.DataFrame()\n",
    "disagreeds = pd.DataFrame()\n",
    "\n",
    "#check_oofs = True\n",
    "check_oofs = False\n",
    "\n",
    "\n",
    "if check_oofs:\n",
    "    for i, oof in enumerate(oofs):\n",
    "        agreeds['oofs_agreed_{}'.format(i)] = oofs[i]['agreed'].values\n",
    "        unrelated['oofs_unrelated_{}'.format(i)] = oofs[i]['unrelated'].values\n",
    "        disagreeds['oofs_disagreeds_{}'.format(i)] = oofs[i]['disagreed'].values\n",
    "else:\n",
    "    for i, oof in enumerate(oofs):\n",
    "        agreeds['oofs_agreed_{}'.format(i)] = preds[i]['agreed'].values\n",
    "        unrelated['oofs_unrelated_{}'.format(i)] = preds[i]['unrelated'].values\n",
    "        disagreeds['oofs_disagreeds_{}'.format(i)] = preds[i]['disagreed'].values  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>oofs_agreed_0</th>\n",
       "      <th>oofs_agreed_1</th>\n",
       "      <th>oofs_agreed_2</th>\n",
       "      <th>oofs_agreed_3</th>\n",
       "      <th>oofs_agreed_4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>oofs_agreed_0</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.999462</td>\n",
       "      <td>0.998657</td>\n",
       "      <td>0.995063</td>\n",
       "      <td>0.998853</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_agreed_1</th>\n",
       "      <td>0.999462</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.999139</td>\n",
       "      <td>0.994128</td>\n",
       "      <td>0.998696</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_agreed_2</th>\n",
       "      <td>0.998657</td>\n",
       "      <td>0.999139</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.992774</td>\n",
       "      <td>0.997810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_agreed_3</th>\n",
       "      <td>0.995063</td>\n",
       "      <td>0.994128</td>\n",
       "      <td>0.992774</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.995594</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_agreed_4</th>\n",
       "      <td>0.998853</td>\n",
       "      <td>0.998696</td>\n",
       "      <td>0.997810</td>\n",
       "      <td>0.995594</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               oofs_agreed_0  oofs_agreed_1  oofs_agreed_2  oofs_agreed_3  \\\n",
       "oofs_agreed_0       1.000000       0.999462       0.998657       0.995063   \n",
       "oofs_agreed_1       0.999462       1.000000       0.999139       0.994128   \n",
       "oofs_agreed_2       0.998657       0.999139       1.000000       0.992774   \n",
       "oofs_agreed_3       0.995063       0.994128       0.992774       1.000000   \n",
       "oofs_agreed_4       0.998853       0.998696       0.997810       0.995594   \n",
       "\n",
       "               oofs_agreed_4  \n",
       "oofs_agreed_0       0.998853  \n",
       "oofs_agreed_1       0.998696  \n",
       "oofs_agreed_2       0.997810  \n",
       "oofs_agreed_3       0.995594  \n",
       "oofs_agreed_4       1.000000  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agreeds.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>oofs_disagreeds_0</th>\n",
       "      <th>oofs_disagreeds_1</th>\n",
       "      <th>oofs_disagreeds_2</th>\n",
       "      <th>oofs_disagreeds_3</th>\n",
       "      <th>oofs_disagreeds_4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>oofs_disagreeds_0</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.995894</td>\n",
       "      <td>0.995529</td>\n",
       "      <td>0.954959</td>\n",
       "      <td>0.934220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_disagreeds_1</th>\n",
       "      <td>0.995894</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.997784</td>\n",
       "      <td>0.948563</td>\n",
       "      <td>0.926715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_disagreeds_2</th>\n",
       "      <td>0.995529</td>\n",
       "      <td>0.997784</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.948111</td>\n",
       "      <td>0.926025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_disagreeds_3</th>\n",
       "      <td>0.954959</td>\n",
       "      <td>0.948563</td>\n",
       "      <td>0.948111</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.925313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_disagreeds_4</th>\n",
       "      <td>0.934220</td>\n",
       "      <td>0.926715</td>\n",
       "      <td>0.926025</td>\n",
       "      <td>0.925313</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   oofs_disagreeds_0  oofs_disagreeds_1  oofs_disagreeds_2  \\\n",
       "oofs_disagreeds_0           1.000000           0.995894           0.995529   \n",
       "oofs_disagreeds_1           0.995894           1.000000           0.997784   \n",
       "oofs_disagreeds_2           0.995529           0.997784           1.000000   \n",
       "oofs_disagreeds_3           0.954959           0.948563           0.948111   \n",
       "oofs_disagreeds_4           0.934220           0.926715           0.926025   \n",
       "\n",
       "                   oofs_disagreeds_3  oofs_disagreeds_4  \n",
       "oofs_disagreeds_0           0.954959           0.934220  \n",
       "oofs_disagreeds_1           0.948563           0.926715  \n",
       "oofs_disagreeds_2           0.948111           0.926025  \n",
       "oofs_disagreeds_3           1.000000           0.925313  \n",
       "oofs_disagreeds_4           0.925313           1.000000  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "disagreeds.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>oofs_unrelated_0</th>\n",
       "      <th>oofs_unrelated_1</th>\n",
       "      <th>oofs_unrelated_2</th>\n",
       "      <th>oofs_unrelated_3</th>\n",
       "      <th>oofs_unrelated_4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>oofs_unrelated_0</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.999235</td>\n",
       "      <td>0.998382</td>\n",
       "      <td>0.992162</td>\n",
       "      <td>0.998741</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_unrelated_1</th>\n",
       "      <td>0.999235</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.998972</td>\n",
       "      <td>0.990868</td>\n",
       "      <td>0.998143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_unrelated_2</th>\n",
       "      <td>0.998382</td>\n",
       "      <td>0.998972</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.989487</td>\n",
       "      <td>0.997331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_unrelated_3</th>\n",
       "      <td>0.992162</td>\n",
       "      <td>0.990868</td>\n",
       "      <td>0.989487</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.990611</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_unrelated_4</th>\n",
       "      <td>0.998741</td>\n",
       "      <td>0.998143</td>\n",
       "      <td>0.997331</td>\n",
       "      <td>0.990611</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  oofs_unrelated_0  oofs_unrelated_1  oofs_unrelated_2  \\\n",
       "oofs_unrelated_0          1.000000          0.999235          0.998382   \n",
       "oofs_unrelated_1          0.999235          1.000000          0.998972   \n",
       "oofs_unrelated_2          0.998382          0.998972          1.000000   \n",
       "oofs_unrelated_3          0.992162          0.990868          0.989487   \n",
       "oofs_unrelated_4          0.998741          0.998143          0.997331   \n",
       "\n",
       "                  oofs_unrelated_3  oofs_unrelated_4  \n",
       "oofs_unrelated_0          0.992162          0.998741  \n",
       "oofs_unrelated_1          0.990868          0.998143  \n",
       "oofs_unrelated_2          0.989487          0.997331  \n",
       "oofs_unrelated_3          1.000000          0.990611  \n",
       "oofs_unrelated_4          0.990611          1.000000  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unrelated.corr()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare Different Inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Only use oofs\n",
    "ensemble_trains = trains.values\n",
    "ensemble_tests = tests.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# use oofs and meta-features\n",
    "#ensemble_trains = np.concatenate((trains.values, trick_trains_features), axis=1)\n",
    "#ensemble_tests = np.concatenate((tests.values, trick_tests_features), axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ensemble With NN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "########################################\n",
    "## import packages\n",
    "########################################\n",
    "import os\n",
    "import re\n",
    "import csv\n",
    "import codecs\n",
    "import numpy as np\n",
    "np.random.seed(1337)\n",
    "\n",
    "import pandas as pd\n",
    "import operator\n",
    "import sys\n",
    "\n",
    "from string import punctuation\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "from iwillwin.trainer.supervised_trainer import KerasModelTrainer\n",
    "from iwillwin.data_utils.data_helpers import DataTransformer, DataLoader\n",
    "from iwillwin.model.sim_zoos import *\n",
    "import tensorflow as tf\n",
    "from keras.layers import Dense, Input, MaxPooling1D, CuDNNLSTM, Embedding, Add, Lambda, Dropout, Activation, SpatialDropout1D, Reshape, GlobalAveragePooling1D, merge, Flatten, Bidirectional, CuDNNGRU, add, Conv1D, GlobalMaxPooling1D\n",
    "from keras.layers.merge import concatenate\n",
    "from keras.models import Model\n",
    "from keras import optimizers\n",
    "from keras import initializers\n",
    "from keras.engine import InputSpec, Layer\n",
    "from iwillwin.config import dataset_config, model_config\n",
    "from keras.models import Sequential\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.layers.core import Lambda, Dense, Dropout\n",
    "from keras.layers.recurrent import LSTM, GRU\n",
    "from keras.layers.wrappers import Bidirectional\n",
    "from keras.legacy.layers import Highway\n",
    "from keras.layers import TimeDistributed\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "import keras.backend as K\n",
    "\n",
    "from sklearn.metrics import roc_auc_score, log_loss\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "from sklearn.metrics import log_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras import regularizers\n",
    "\n",
    "def weighted_accuracy(y_true, y_pred):\n",
    "    weight = np.array([[1/16, 1/15, 1/5]])\n",
    "    norm = [(1/16) + (1/15) + (1/5)]\n",
    "    weight_mask = weight * y_true\n",
    "    \n",
    "    y_pred = K.cast(y_pred > 0.5, 'int32') # Hard\n",
    "    y_true = K.cast(y_true, 'int32')\n",
    "    \n",
    "    res = K.cast(K.equal(y_pred, y_true), 'float32') * weight_mask / K.sum(weight_mask)\n",
    "    res = K.sum(res)\n",
    "    return res\n",
    "\n",
    "def get_dense_add_net(feature_nums):\n",
    "    features_inputs = Input(shape=(feature_nums,), name='mata-features', dtype=\"float32\")\n",
    "    features = features_inputs\n",
    "    \n",
    "    depth = 5\n",
    "    for i in range(depth):\n",
    "        new_features = Dense(24, activation='relu')(features)\n",
    "        new_features = Dropout(0.1)(new_features)\n",
    "        features = Concatenate()([features, new_features])\n",
    "\n",
    "    h = Highway(activation='relu')(features)\n",
    "    out_ = Dense(3, activation='softmax')(h)\n",
    "    \n",
    "    model = Model(inputs=[features_inputs], outputs=out_)\n",
    "    model.compile(optimizer=Adam(lr=1e-3, decay=1e-6,), loss='categorical_crossentropy',\n",
    "    metrics=['accuracy', weighted_accuracy])\n",
    "    model.summary()\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_linear_net(feature_nums):\n",
    "    features_inputs = Input(shape=(feature_nums,), name='mata-features', dtype=\"float32\")    \n",
    "    out_ = Dense(3, activation='softmax')(features_inputs)\n",
    "    model = Model(inputs=[features_inputs], outputs=out_)\n",
    "    model.compile(optimizer=Adam(lr=1e-3, decay=1e-6,), loss='categorical_crossentropy',\n",
    "    metrics=['accuracy', weighted_accuracy])\n",
    "    model.summary()\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "import importlib\n",
    "\n",
    "from sklearn.metrics import roc_auc_score, log_loss\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "\n",
    "from iwillwin.config import model_config\n",
    "\n",
    "class ModelTrainer(object):\n",
    "\n",
    "    def __init__(self, model_stamp, epoch_num, learning_rate=1e-3,\n",
    "                 shuffle_inputs=False, verbose_round=40, early_stopping_round=8):\n",
    "        self.models = []\n",
    "        self.model_stamp = model_stamp\n",
    "        self.val_loss = -1\n",
    "        self.auc = -1\n",
    "        self.epoch_num = epoch_num\n",
    "        self.learning_rate = learning_rate\n",
    "        self.eps = 1e-10\n",
    "        self.verbose_round = verbose_round\n",
    "        self.early_stopping_round = early_stopping_round\n",
    "        self.shuffle_inputs = shuffle_inputs\n",
    "        self.class_weight = [0.93, 1.21]\n",
    "\n",
    "    def train_folds(self, features, y, fold_count, batch_size, get_model_func, augments=None, skip_fold=0, patience=10, scale_sample_weight=False,\n",
    "                    class_weight=None, self_aware=False, swap_input=False):\n",
    "        weight_val=scale_sample_weight\n",
    "        class_weight=None\n",
    "        fold_size = len(features) // fold_count\n",
    "        models = []\n",
    "        fold_predictions = []\n",
    "        score = 0\n",
    "\n",
    "        for fold_id in range(0, fold_count):\n",
    "            fold_start = fold_size * fold_id\n",
    "            fold_end = fold_start + fold_size\n",
    "\n",
    "            if fold_id == fold_count - 1:\n",
    "                fold_end = len(features)\n",
    "\n",
    "            train_features = np.concatenate([features[:fold_start], features[fold_end:]])\n",
    "            train_y = np.concatenate([y[:fold_start], y[fold_end:]])\n",
    "            \n",
    "            val_features = features[fold_start:fold_end]\n",
    "            val_y = y[fold_start:fold_end]\n",
    "            fold_pos = (np.sum(train_y) / len(train_features))\n",
    "\n",
    "            train_data = {\n",
    "                \"mata-features\": train_features,\n",
    "            }\n",
    "\n",
    "            val_data = {\n",
    "                \"mata-features\": val_features,\n",
    "            }\n",
    "\n",
    "            model, bst_val_score, fold_prediction = self._train_model_by_logloss(\n",
    "                get_model_func(), batch_size, train_data, train_y, val_data, val_y, fold_id, patience, class_weight, weight_val=None)\n",
    "    \n",
    "            score += bst_val_score\n",
    "            models.append(model)\n",
    "            fold_predictions.append(fold_prediction)\n",
    "\n",
    "        self.models = models\n",
    "        self.val_loss = score / fold_count\n",
    "        return models, self.val_loss, fold_predictions\n",
    "\n",
    "    def _train_model_by_logloss(self, model, batch_size, train_x, train_y, val_x, val_y, fold_id, patience):\n",
    "        # return a list which holds [models, val_loss, auc, prediction]\n",
    "        raise NotImplementedError\n",
    "\n",
    "class KerasModelTrainer(ModelTrainer):\n",
    "\n",
    "    def __init__(self, *args, **kwargs):\n",
    "        super(KerasModelTrainer, self).__init__(*args, **kwargs)\n",
    "        pass\n",
    "\n",
    "    def _train_model_by_logloss(self, model, batch_size, train_x, train_y, val_x, val_y, fold_id, patience, class_weight, weight_val):\n",
    "        early_stopping = EarlyStopping(monitor='val_loss', patience=10)\n",
    "        bst_model_path = self.model_stamp + str(fold_id) + '.h5'\n",
    "        val_data = (val_x, val_y, weight_val) if weight_val is not None else (val_x, val_y)\n",
    "        model_checkpoint = ModelCheckpoint(bst_model_path, save_best_only=True, save_weights_only=True)\n",
    "        hist = model.fit(train_x, train_y,\n",
    "                         validation_data=val_data,\n",
    "                         epochs=self.epoch_num, batch_size=batch_size, shuffle=True,\n",
    "                         verbose=2,\n",
    "                         class_weight={0: 1/16, 1: 1/15, 2: 1/5},\n",
    "                         callbacks=[early_stopping, model_checkpoint],)\n",
    "        bst_val_score = max(hist.history['val_weighted_accuracy'])\n",
    "        model.load_weights(bst_model_path)\n",
    "        predictions = model.predict(val_x)\n",
    "\n",
    "        return model, bst_val_score, predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def _agent_get_model():\n",
    "    return get_dense_add_net(ensemble_trains.shape[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def np_weighted_accuracy(y_true, y_pred):\n",
    "    weight = np.array([[1/16, 1/15, 1/5]])\n",
    "    norm = [(1/16) + (1/15) + (1/5)]\n",
    "    weight_mask = weight * y_true\n",
    "    weight_mask = np.max(weight_mask, axis=-1)\n",
    "    norms = np.sum(weight_mask)\n",
    "    \n",
    "    y_true = np.argmax(y_true, axis=-1)\n",
    "    y_pred = np.argmax(y_pred, axis=-1)\n",
    "    \n",
    "    res = ((y_true == y_pred) * weight_mask).sum() / norms\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def weighted_accuracy(y_true, y_pred):\n",
    "    weight = np.array([[1/16, 1/15, 1/5]])\n",
    "    norm = [(1/16) + (1/15) + (1/5)]\n",
    "    weight_mask = weight * y_true\n",
    "    \n",
    "    y_pred = K.cast(y_pred > 0.5, 'int32') # Hard\n",
    "    y_true = K.cast(y_true, 'int32')\n",
    "    \n",
    "    res = K.cast(K.equal(y_pred, y_true), 'float32') * weight_mask / K.sum(weight_mask)\n",
    "    res = K.sum(res)\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\keras\\legacy\\layers.py:198: UserWarning: The `Highway` layer is deprecated and will be removed after 06/2017.\n",
      "  warnings.warn('The `Highway` layer is deprecated '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "mata-features (InputLayer)      (None, 15)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_73 (Dense)                (None, 24)           384         mata-features[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_61 (Dropout)            (None, 24)           0           dense_73[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_61 (Concatenate)    (None, 39)           0           mata-features[0][0]              \n",
      "                                                                 dropout_61[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_74 (Dense)                (None, 24)           960         concatenate_61[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_62 (Dropout)            (None, 24)           0           dense_74[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_62 (Concatenate)    (None, 63)           0           concatenate_61[0][0]             \n",
      "                                                                 dropout_62[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_75 (Dense)                (None, 24)           1536        concatenate_62[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_63 (Dropout)            (None, 24)           0           dense_75[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_63 (Concatenate)    (None, 87)           0           concatenate_62[0][0]             \n",
      "                                                                 dropout_63[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_76 (Dense)                (None, 24)           2112        concatenate_63[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_64 (Dropout)            (None, 24)           0           dense_76[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_64 (Concatenate)    (None, 111)          0           concatenate_63[0][0]             \n",
      "                                                                 dropout_64[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_77 (Dense)                (None, 24)           2688        concatenate_64[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_65 (Dropout)            (None, 24)           0           dense_77[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_65 (Concatenate)    (None, 135)          0           concatenate_64[0][0]             \n",
      "                                                                 dropout_65[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "highway_13 (Highway)            (None, 135)          36720       concatenate_65[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dense_78 (Dense)                (None, 3)            408         highway_13[0][0]                 \n",
      "==================================================================================================\n",
      "Total params: 44,808\n",
      "Trainable params: 44,808\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Train on 288497 samples, validate on 32055 samples\n",
      "Epoch 1/500\n",
      " - 2s - loss: 0.0227 - acc: 0.8747 - weighted_accuracy: 0.8483 - val_loss: 0.2940 - val_acc: 0.8710 - val_weighted_accuracy: 0.8635\n",
      "Epoch 2/500\n",
      " - 1s - loss: 0.0201 - acc: 0.8779 - weighted_accuracy: 0.8691 - val_loss: 0.2846 - val_acc: 0.8734 - val_weighted_accuracy: 0.8645\n",
      "Epoch 3/500\n",
      " - 2s - loss: 0.0200 - acc: 0.8779 - weighted_accuracy: 0.8691 - val_loss: 0.2780 - val_acc: 0.8774 - val_weighted_accuracy: 0.8659\n",
      "Epoch 4/500\n",
      " - 1s - loss: 0.0199 - acc: 0.8779 - weighted_accuracy: 0.8694 - val_loss: 0.2812 - val_acc: 0.8763 - val_weighted_accuracy: 0.8658\n",
      "Epoch 5/500\n",
      " - 1s - loss: 0.0199 - acc: 0.8780 - weighted_accuracy: 0.8692 - val_loss: 0.2830 - val_acc: 0.8741 - val_weighted_accuracy: 0.8646\n",
      "Epoch 6/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8780 - weighted_accuracy: 0.8692 - val_loss: 0.2861 - val_acc: 0.8734 - val_weighted_accuracy: 0.8646\n",
      "Epoch 7/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8782 - weighted_accuracy: 0.8697 - val_loss: 0.2807 - val_acc: 0.8742 - val_weighted_accuracy: 0.8651\n",
      "Epoch 8/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8778 - weighted_accuracy: 0.8693 - val_loss: 0.2873 - val_acc: 0.8724 - val_weighted_accuracy: 0.8643\n",
      "Epoch 9/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8780 - weighted_accuracy: 0.8692 - val_loss: 0.2818 - val_acc: 0.8743 - val_weighted_accuracy: 0.8651\n",
      "Epoch 10/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8778 - weighted_accuracy: 0.8695 - val_loss: 0.2845 - val_acc: 0.8757 - val_weighted_accuracy: 0.8657\n",
      "Epoch 11/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8778 - weighted_accuracy: 0.8693 - val_loss: 0.2893 - val_acc: 0.8722 - val_weighted_accuracy: 0.8642\n",
      "Epoch 12/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8782 - weighted_accuracy: 0.8696 - val_loss: 0.2874 - val_acc: 0.8730 - val_weighted_accuracy: 0.8647\n",
      "Epoch 13/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8782 - weighted_accuracy: 0.8695 - val_loss: 0.2824 - val_acc: 0.8740 - val_weighted_accuracy: 0.8650\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "mata-features (InputLayer)      (None, 15)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_79 (Dense)                (None, 24)           384         mata-features[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_66 (Dropout)            (None, 24)           0           dense_79[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_66 (Concatenate)    (None, 39)           0           mata-features[0][0]              \n",
      "                                                                 dropout_66[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_80 (Dense)                (None, 24)           960         concatenate_66[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_67 (Dropout)            (None, 24)           0           dense_80[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_67 (Concatenate)    (None, 63)           0           concatenate_66[0][0]             \n",
      "                                                                 dropout_67[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_81 (Dense)                (None, 24)           1536        concatenate_67[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_68 (Dropout)            (None, 24)           0           dense_81[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_68 (Concatenate)    (None, 87)           0           concatenate_67[0][0]             \n",
      "                                                                 dropout_68[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_82 (Dense)                (None, 24)           2112        concatenate_68[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_69 (Dropout)            (None, 24)           0           dense_82[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_69 (Concatenate)    (None, 111)          0           concatenate_68[0][0]             \n",
      "                                                                 dropout_69[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_83 (Dense)                (None, 24)           2688        concatenate_69[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_70 (Dropout)            (None, 24)           0           dense_83[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_70 (Concatenate)    (None, 135)          0           concatenate_69[0][0]             \n",
      "                                                                 dropout_70[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "highway_14 (Highway)            (None, 135)          36720       concatenate_70[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dense_84 (Dense)                (None, 3)            408         highway_14[0][0]                 \n",
      "==================================================================================================\n",
      "Total params: 44,808\n",
      "Trainable params: 44,808\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Train on 288497 samples, validate on 32055 samples\n",
      "Epoch 1/500\n",
      " - 2s - loss: 0.0229 - acc: 0.8702 - weighted_accuracy: 0.8481 - val_loss: 0.2599 - val_acc: 0.8846 - val_weighted_accuracy: 0.8798\n",
      "Epoch 2/500\n",
      " - 1s - loss: 0.0205 - acc: 0.8764 - weighted_accuracy: 0.8669 - val_loss: 0.2526 - val_acc: 0.8859 - val_weighted_accuracy: 0.8785\n",
      "Epoch 3/500\n",
      " - 1s - loss: 0.0203 - acc: 0.8764 - weighted_accuracy: 0.8673 - val_loss: 0.2446 - val_acc: 0.8919 - val_weighted_accuracy: 0.8819\n",
      "Epoch 4/500\n",
      " - 2s - loss: 0.0202 - acc: 0.8767 - weighted_accuracy: 0.8677 - val_loss: 0.2455 - val_acc: 0.8906 - val_weighted_accuracy: 0.8803\n",
      "Epoch 5/500\n",
      " - 1s - loss: 0.0202 - acc: 0.8762 - weighted_accuracy: 0.8675 - val_loss: 0.2505 - val_acc: 0.8872 - val_weighted_accuracy: 0.8785\n",
      "Epoch 6/500\n",
      " - 1s - loss: 0.0202 - acc: 0.8766 - weighted_accuracy: 0.8675 - val_loss: 0.2506 - val_acc: 0.8883 - val_weighted_accuracy: 0.8805\n",
      "Epoch 7/500\n",
      " - 1s - loss: 0.0201 - acc: 0.8764 - weighted_accuracy: 0.8675 - val_loss: 0.2442 - val_acc: 0.8918 - val_weighted_accuracy: 0.8817\n",
      "Epoch 8/500\n",
      " - 1s - loss: 0.0201 - acc: 0.8766 - weighted_accuracy: 0.8674 - val_loss: 0.2474 - val_acc: 0.8898 - val_weighted_accuracy: 0.8807\n",
      "Epoch 9/500\n",
      " - 2s - loss: 0.0201 - acc: 0.8769 - weighted_accuracy: 0.8679 - val_loss: 0.2458 - val_acc: 0.8895 - val_weighted_accuracy: 0.8818\n",
      "Epoch 10/500\n",
      " - 1s - loss: 0.0201 - acc: 0.8764 - weighted_accuracy: 0.8675 - val_loss: 0.2499 - val_acc: 0.8893 - val_weighted_accuracy: 0.8816\n",
      "Epoch 11/500\n",
      " - 1s - loss: 0.0201 - acc: 0.8769 - weighted_accuracy: 0.8680 - val_loss: 0.2497 - val_acc: 0.8894 - val_weighted_accuracy: 0.8813\n",
      "Epoch 12/500\n",
      " - 2s - loss: 0.0201 - acc: 0.8769 - weighted_accuracy: 0.8678 - val_loss: 0.2465 - val_acc: 0.8895 - val_weighted_accuracy: 0.8810\n",
      "Epoch 13/500\n",
      " - 1s - loss: 0.0201 - acc: 0.8769 - weighted_accuracy: 0.8679 - val_loss: 0.2446 - val_acc: 0.8900 - val_weighted_accuracy: 0.8807\n",
      "Epoch 14/500\n",
      " - 1s - loss: 0.0201 - acc: 0.8768 - weighted_accuracy: 0.8677 - val_loss: 0.2433 - val_acc: 0.8908 - val_weighted_accuracy: 0.8810\n",
      "Epoch 15/500\n",
      " - 1s - loss: 0.0200 - acc: 0.8768 - weighted_accuracy: 0.8678 - val_loss: 0.2488 - val_acc: 0.8882 - val_weighted_accuracy: 0.8806\n",
      "Epoch 16/500\n",
      " - 1s - loss: 0.0201 - acc: 0.8766 - weighted_accuracy: 0.8676 - val_loss: 0.2490 - val_acc: 0.8906 - val_weighted_accuracy: 0.8809\n",
      "Epoch 17/500\n",
      " - 1s - loss: 0.0200 - acc: 0.8769 - weighted_accuracy: 0.8678 - val_loss: 0.2494 - val_acc: 0.8901 - val_weighted_accuracy: 0.8817\n",
      "Epoch 18/500\n",
      " - 1s - loss: 0.0200 - acc: 0.8769 - weighted_accuracy: 0.8679 - val_loss: 0.2472 - val_acc: 0.8889 - val_weighted_accuracy: 0.8807\n",
      "Epoch 19/500\n",
      " - 1s - loss: 0.0200 - acc: 0.8766 - weighted_accuracy: 0.8677 - val_loss: 0.2475 - val_acc: 0.8891 - val_weighted_accuracy: 0.8812\n",
      "Epoch 20/500\n",
      " - 1s - loss: 0.0200 - acc: 0.8768 - weighted_accuracy: 0.8679 - val_loss: 0.2429 - val_acc: 0.8911 - val_weighted_accuracy: 0.8808\n",
      "Epoch 21/500\n",
      " - 2s - loss: 0.0200 - acc: 0.8769 - weighted_accuracy: 0.8678 - val_loss: 0.2447 - val_acc: 0.8903 - val_weighted_accuracy: 0.8814\n",
      "Epoch 22/500\n",
      " - 1s - loss: 0.0200 - acc: 0.8770 - weighted_accuracy: 0.8679 - val_loss: 0.2436 - val_acc: 0.8912 - val_weighted_accuracy: 0.8811\n",
      "Epoch 23/500\n",
      " - 1s - loss: 0.0200 - acc: 0.8766 - weighted_accuracy: 0.8675 - val_loss: 0.2510 - val_acc: 0.8894 - val_weighted_accuracy: 0.8813\n",
      "Epoch 24/500\n",
      " - 1s - loss: 0.0200 - acc: 0.8765 - weighted_accuracy: 0.8676 - val_loss: 0.2485 - val_acc: 0.8895 - val_weighted_accuracy: 0.8808\n",
      "Epoch 25/500\n",
      " - 1s - loss: 0.0200 - acc: 0.8772 - weighted_accuracy: 0.8680 - val_loss: 0.2452 - val_acc: 0.8911 - val_weighted_accuracy: 0.8814\n",
      "Epoch 26/500\n",
      " - 1s - loss: 0.0200 - acc: 0.8768 - weighted_accuracy: 0.8678 - val_loss: 0.2425 - val_acc: 0.8911 - val_weighted_accuracy: 0.8808\n",
      "Epoch 27/500\n",
      " - 2s - loss: 0.0200 - acc: 0.8770 - weighted_accuracy: 0.8679 - val_loss: 0.2449 - val_acc: 0.8907 - val_weighted_accuracy: 0.8820\n",
      "Epoch 28/500\n",
      " - 1s - loss: 0.0200 - acc: 0.8767 - weighted_accuracy: 0.8678 - val_loss: 0.2427 - val_acc: 0.8922 - val_weighted_accuracy: 0.8819\n",
      "Epoch 29/500\n",
      " - 1s - loss: 0.0200 - acc: 0.8768 - weighted_accuracy: 0.8680 - val_loss: 0.2462 - val_acc: 0.8897 - val_weighted_accuracy: 0.8812\n",
      "Epoch 30/500\n",
      " - 1s - loss: 0.0200 - acc: 0.8767 - weighted_accuracy: 0.8679 - val_loss: 0.2465 - val_acc: 0.8888 - val_weighted_accuracy: 0.8811\n",
      "Epoch 31/500\n",
      " - 1s - loss: 0.0200 - acc: 0.8767 - weighted_accuracy: 0.8677 - val_loss: 0.2472 - val_acc: 0.8892 - val_weighted_accuracy: 0.8817\n",
      "Epoch 32/500\n",
      " - 1s - loss: 0.0200 - acc: 0.8770 - weighted_accuracy: 0.8680 - val_loss: 0.2483 - val_acc: 0.8898 - val_weighted_accuracy: 0.8815\n",
      "Epoch 33/500\n",
      " - 1s - loss: 0.0200 - acc: 0.8766 - weighted_accuracy: 0.8678 - val_loss: 0.2442 - val_acc: 0.8913 - val_weighted_accuracy: 0.8820\n",
      "Epoch 34/500\n",
      " - 1s - loss: 0.0200 - acc: 0.8767 - weighted_accuracy: 0.8678 - val_loss: 0.2444 - val_acc: 0.8915 - val_weighted_accuracy: 0.8815\n",
      "Epoch 35/500\n",
      " - 1s - loss: 0.0200 - acc: 0.8767 - weighted_accuracy: 0.8678 - val_loss: 0.2433 - val_acc: 0.8914 - val_weighted_accuracy: 0.8817\n",
      "Epoch 36/500\n",
      " - 1s - loss: 0.0200 - acc: 0.8769 - weighted_accuracy: 0.8679 - val_loss: 0.2450 - val_acc: 0.8907 - val_weighted_accuracy: 0.8818\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "mata-features (InputLayer)      (None, 15)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_85 (Dense)                (None, 24)           384         mata-features[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_71 (Dropout)            (None, 24)           0           dense_85[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_71 (Concatenate)    (None, 39)           0           mata-features[0][0]              \n",
      "                                                                 dropout_71[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_86 (Dense)                (None, 24)           960         concatenate_71[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_72 (Dropout)            (None, 24)           0           dense_86[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_72 (Concatenate)    (None, 63)           0           concatenate_71[0][0]             \n",
      "                                                                 dropout_72[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_87 (Dense)                (None, 24)           1536        concatenate_72[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_73 (Dropout)            (None, 24)           0           dense_87[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_73 (Concatenate)    (None, 87)           0           concatenate_72[0][0]             \n",
      "                                                                 dropout_73[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_88 (Dense)                (None, 24)           2112        concatenate_73[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_74 (Dropout)            (None, 24)           0           dense_88[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_74 (Concatenate)    (None, 111)          0           concatenate_73[0][0]             \n",
      "                                                                 dropout_74[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_89 (Dense)                (None, 24)           2688        concatenate_74[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_75 (Dropout)            (None, 24)           0           dense_89[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_75 (Concatenate)    (None, 135)          0           concatenate_74[0][0]             \n",
      "                                                                 dropout_75[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "highway_15 (Highway)            (None, 135)          36720       concatenate_75[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dense_90 (Dense)                (None, 3)            408         highway_15[0][0]                 \n",
      "==================================================================================================\n",
      "Total params: 44,808\n",
      "Trainable params: 44,808\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Train on 288497 samples, validate on 32055 samples\n",
      "Epoch 1/500\n",
      " - 2s - loss: 0.0232 - acc: 0.8674 - weighted_accuracy: 0.8448 - val_loss: 0.2728 - val_acc: 0.8800 - val_weighted_accuracy: 0.8709\n",
      "Epoch 2/500\n",
      " - 1s - loss: 0.0203 - acc: 0.8775 - weighted_accuracy: 0.8683 - val_loss: 0.2719 - val_acc: 0.8813 - val_weighted_accuracy: 0.8723\n",
      "Epoch 3/500\n",
      " - 1s - loss: 0.0201 - acc: 0.8780 - weighted_accuracy: 0.8688 - val_loss: 0.2712 - val_acc: 0.8814 - val_weighted_accuracy: 0.8722\n",
      "Epoch 4/500\n",
      " - 2s - loss: 0.0201 - acc: 0.8780 - weighted_accuracy: 0.8687 - val_loss: 0.2721 - val_acc: 0.8794 - val_weighted_accuracy: 0.8715\n",
      "Epoch 5/500\n",
      " - 2s - loss: 0.0200 - acc: 0.8779 - weighted_accuracy: 0.8685 - val_loss: 0.2690 - val_acc: 0.8812 - val_weighted_accuracy: 0.8720\n",
      "Epoch 6/500\n",
      " - 1s - loss: 0.0200 - acc: 0.8777 - weighted_accuracy: 0.8686 - val_loss: 0.2723 - val_acc: 0.8797 - val_weighted_accuracy: 0.8711\n",
      "Epoch 7/500\n",
      " - 1s - loss: 0.0199 - acc: 0.8779 - weighted_accuracy: 0.8689 - val_loss: 0.2729 - val_acc: 0.8810 - val_weighted_accuracy: 0.8720\n",
      "Epoch 8/500\n",
      " - 1s - loss: 0.0199 - acc: 0.8779 - weighted_accuracy: 0.8688 - val_loss: 0.2679 - val_acc: 0.8820 - val_weighted_accuracy: 0.8709\n",
      "Epoch 9/500\n",
      " - 1s - loss: 0.0199 - acc: 0.8781 - weighted_accuracy: 0.8690 - val_loss: 0.2764 - val_acc: 0.8797 - val_weighted_accuracy: 0.8710\n",
      "Epoch 10/500\n",
      " - 1s - loss: 0.0199 - acc: 0.8782 - weighted_accuracy: 0.8691 - val_loss: 0.2723 - val_acc: 0.8793 - val_weighted_accuracy: 0.8718\n",
      "Epoch 11/500\n",
      " - 1s - loss: 0.0199 - acc: 0.8783 - weighted_accuracy: 0.8691 - val_loss: 0.2726 - val_acc: 0.8780 - val_weighted_accuracy: 0.8711\n",
      "Epoch 12/500\n",
      " - 1s - loss: 0.0199 - acc: 0.8782 - weighted_accuracy: 0.8689 - val_loss: 0.2689 - val_acc: 0.8809 - val_weighted_accuracy: 0.8712\n",
      "Epoch 13/500\n",
      " - 1s - loss: 0.0199 - acc: 0.8779 - weighted_accuracy: 0.8689 - val_loss: 0.2739 - val_acc: 0.8796 - val_weighted_accuracy: 0.8714\n",
      "Epoch 14/500\n",
      " - 1s - loss: 0.0199 - acc: 0.8781 - weighted_accuracy: 0.8691 - val_loss: 0.2680 - val_acc: 0.8824 - val_weighted_accuracy: 0.8712\n",
      "Epoch 15/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8785 - weighted_accuracy: 0.8690 - val_loss: 0.2712 - val_acc: 0.8800 - val_weighted_accuracy: 0.8711\n",
      "Epoch 16/500\n",
      " - 1s - loss: 0.0199 - acc: 0.8783 - weighted_accuracy: 0.8689 - val_loss: 0.2691 - val_acc: 0.8804 - val_weighted_accuracy: 0.8716\n",
      "Epoch 17/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8781 - weighted_accuracy: 0.8689 - val_loss: 0.2749 - val_acc: 0.8768 - val_weighted_accuracy: 0.8702\n",
      "Epoch 18/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8780 - weighted_accuracy: 0.8688 - val_loss: 0.2700 - val_acc: 0.8818 - val_weighted_accuracy: 0.8717\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "mata-features (InputLayer)      (None, 15)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_91 (Dense)                (None, 24)           384         mata-features[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_76 (Dropout)            (None, 24)           0           dense_91[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_76 (Concatenate)    (None, 39)           0           mata-features[0][0]              \n",
      "                                                                 dropout_76[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_92 (Dense)                (None, 24)           960         concatenate_76[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_77 (Dropout)            (None, 24)           0           dense_92[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_77 (Concatenate)    (None, 63)           0           concatenate_76[0][0]             \n",
      "                                                                 dropout_77[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_93 (Dense)                (None, 24)           1536        concatenate_77[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_78 (Dropout)            (None, 24)           0           dense_93[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_78 (Concatenate)    (None, 87)           0           concatenate_77[0][0]             \n",
      "                                                                 dropout_78[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_94 (Dense)                (None, 24)           2112        concatenate_78[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_79 (Dropout)            (None, 24)           0           dense_94[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_79 (Concatenate)    (None, 111)          0           concatenate_78[0][0]             \n",
      "                                                                 dropout_79[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_95 (Dense)                (None, 24)           2688        concatenate_79[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_80 (Dropout)            (None, 24)           0           dense_95[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_80 (Concatenate)    (None, 135)          0           concatenate_79[0][0]             \n",
      "                                                                 dropout_80[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "highway_16 (Highway)            (None, 135)          36720       concatenate_80[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dense_96 (Dense)                (None, 3)            408         highway_16[0][0]                 \n",
      "==================================================================================================\n",
      "Total params: 44,808\n",
      "Trainable params: 44,808\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Train on 288497 samples, validate on 32055 samples\n",
      "Epoch 1/500\n",
      " - 2s - loss: 0.0225 - acc: 0.8760 - weighted_accuracy: 0.8520 - val_loss: 0.2929 - val_acc: 0.8706 - val_weighted_accuracy: 0.8637\n",
      "Epoch 2/500\n",
      " - 1s - loss: 0.0201 - acc: 0.8788 - weighted_accuracy: 0.8692 - val_loss: 0.2944 - val_acc: 0.8676 - val_weighted_accuracy: 0.8630\n",
      "Epoch 3/500\n",
      " - 1s - loss: 0.0199 - acc: 0.8787 - weighted_accuracy: 0.8693 - val_loss: 0.2908 - val_acc: 0.8724 - val_weighted_accuracy: 0.8637\n",
      "Epoch 4/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8788 - weighted_accuracy: 0.8696 - val_loss: 0.2827 - val_acc: 0.8725 - val_weighted_accuracy: 0.8632\n",
      "Epoch 5/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8788 - weighted_accuracy: 0.8696 - val_loss: 0.2851 - val_acc: 0.8730 - val_weighted_accuracy: 0.8632\n",
      "Epoch 6/500\n",
      " - 2s - loss: 0.0198 - acc: 0.8789 - weighted_accuracy: 0.8696 - val_loss: 0.2886 - val_acc: 0.8733 - val_weighted_accuracy: 0.8644\n",
      "Epoch 7/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8790 - weighted_accuracy: 0.8700 - val_loss: 0.2935 - val_acc: 0.8684 - val_weighted_accuracy: 0.8634\n",
      "Epoch 8/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8790 - weighted_accuracy: 0.8697 - val_loss: 0.2872 - val_acc: 0.8707 - val_weighted_accuracy: 0.8639\n",
      "Epoch 9/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8789 - weighted_accuracy: 0.8699 - val_loss: 0.2881 - val_acc: 0.8710 - val_weighted_accuracy: 0.8638\n",
      "Epoch 10/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8789 - weighted_accuracy: 0.8696 - val_loss: 0.2860 - val_acc: 0.8716 - val_weighted_accuracy: 0.8638\n",
      "Epoch 11/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8792 - weighted_accuracy: 0.8699 - val_loss: 0.2829 - val_acc: 0.8730 - val_weighted_accuracy: 0.8634\n",
      "Epoch 12/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8791 - weighted_accuracy: 0.8698 - val_loss: 0.2832 - val_acc: 0.8726 - val_weighted_accuracy: 0.8630\n",
      "Epoch 13/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8793 - weighted_accuracy: 0.8700 - val_loss: 0.2835 - val_acc: 0.8733 - val_weighted_accuracy: 0.8635\n",
      "Epoch 14/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8794 - weighted_accuracy: 0.8700 - val_loss: 0.2835 - val_acc: 0.8734 - val_weighted_accuracy: 0.8643\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "mata-features (InputLayer)      (None, 15)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_97 (Dense)                (None, 24)           384         mata-features[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_81 (Dropout)            (None, 24)           0           dense_97[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_81 (Concatenate)    (None, 39)           0           mata-features[0][0]              \n",
      "                                                                 dropout_81[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_98 (Dense)                (None, 24)           960         concatenate_81[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_82 (Dropout)            (None, 24)           0           dense_98[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_82 (Concatenate)    (None, 63)           0           concatenate_81[0][0]             \n",
      "                                                                 dropout_82[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_99 (Dense)                (None, 24)           1536        concatenate_82[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_83 (Dropout)            (None, 24)           0           dense_99[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_83 (Concatenate)    (None, 87)           0           concatenate_82[0][0]             \n",
      "                                                                 dropout_83[0][0]                 \n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dense_100 (Dense)               (None, 24)           2112        concatenate_83[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_84 (Dropout)            (None, 24)           0           dense_100[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_84 (Concatenate)    (None, 111)          0           concatenate_83[0][0]             \n",
      "                                                                 dropout_84[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_101 (Dense)               (None, 24)           2688        concatenate_84[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_85 (Dropout)            (None, 24)           0           dense_101[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_85 (Concatenate)    (None, 135)          0           concatenate_84[0][0]             \n",
      "                                                                 dropout_85[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "highway_17 (Highway)            (None, 135)          36720       concatenate_85[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dense_102 (Dense)               (None, 3)            408         highway_17[0][0]                 \n",
      "==================================================================================================\n",
      "Total params: 44,808\n",
      "Trainable params: 44,808\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Train on 288497 samples, validate on 32055 samples\n",
      "Epoch 1/500\n",
      " - 2s - loss: 0.0228 - acc: 0.8699 - weighted_accuracy: 0.8487 - val_loss: 0.2664 - val_acc: 0.8861 - val_weighted_accuracy: 0.8827\n",
      "Epoch 2/500\n",
      " - 1s - loss: 0.0204 - acc: 0.8768 - weighted_accuracy: 0.8675 - val_loss: 0.2650 - val_acc: 0.8891 - val_weighted_accuracy: 0.8833\n",
      "Epoch 3/500\n",
      " - 1s - loss: 0.0203 - acc: 0.8766 - weighted_accuracy: 0.8673 - val_loss: 0.2548 - val_acc: 0.8919 - val_weighted_accuracy: 0.8833\n",
      "Epoch 4/500\n",
      " - 2s - loss: 0.0202 - acc: 0.8767 - weighted_accuracy: 0.8672 - val_loss: 0.2602 - val_acc: 0.8880 - val_weighted_accuracy: 0.8829\n",
      "Epoch 5/500\n",
      " - 1s - loss: 0.0201 - acc: 0.8767 - weighted_accuracy: 0.8676 - val_loss: 0.2627 - val_acc: 0.8885 - val_weighted_accuracy: 0.8831\n",
      "Epoch 6/500\n",
      " - 1s - loss: 0.0201 - acc: 0.8765 - weighted_accuracy: 0.8674 - val_loss: 0.2558 - val_acc: 0.8906 - val_weighted_accuracy: 0.8836\n",
      "Epoch 7/500\n",
      " - 1s - loss: 0.0201 - acc: 0.8767 - weighted_accuracy: 0.8675 - val_loss: 0.2604 - val_acc: 0.8884 - val_weighted_accuracy: 0.8832\n",
      "Epoch 8/500\n",
      " - 1s - loss: 0.0201 - acc: 0.8768 - weighted_accuracy: 0.8674 - val_loss: 0.2567 - val_acc: 0.8905 - val_weighted_accuracy: 0.8837\n",
      "Epoch 9/500\n",
      " - 1s - loss: 0.0200 - acc: 0.8763 - weighted_accuracy: 0.8672 - val_loss: 0.2573 - val_acc: 0.8894 - val_weighted_accuracy: 0.8831\n",
      "Epoch 10/500\n",
      " - 1s - loss: 0.0200 - acc: 0.8766 - weighted_accuracy: 0.8674 - val_loss: 0.2628 - val_acc: 0.8883 - val_weighted_accuracy: 0.8827\n",
      "Epoch 11/500\n",
      " - 1s - loss: 0.0200 - acc: 0.8767 - weighted_accuracy: 0.8675 - val_loss: 0.2556 - val_acc: 0.8919 - val_weighted_accuracy: 0.8833\n",
      "Epoch 12/500\n",
      " - 1s - loss: 0.0200 - acc: 0.8768 - weighted_accuracy: 0.8675 - val_loss: 0.2602 - val_acc: 0.8891 - val_weighted_accuracy: 0.8810\n",
      "Epoch 13/500\n",
      " - 1s - loss: 0.0200 - acc: 0.8769 - weighted_accuracy: 0.8677 - val_loss: 0.2591 - val_acc: 0.8889 - val_weighted_accuracy: 0.8829\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "mata-features (InputLayer)      (None, 15)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_103 (Dense)               (None, 24)           384         mata-features[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_86 (Dropout)            (None, 24)           0           dense_103[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_86 (Concatenate)    (None, 39)           0           mata-features[0][0]              \n",
      "                                                                 dropout_86[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_104 (Dense)               (None, 24)           960         concatenate_86[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_87 (Dropout)            (None, 24)           0           dense_104[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_87 (Concatenate)    (None, 63)           0           concatenate_86[0][0]             \n",
      "                                                                 dropout_87[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_105 (Dense)               (None, 24)           1536        concatenate_87[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_88 (Dropout)            (None, 24)           0           dense_105[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_88 (Concatenate)    (None, 87)           0           concatenate_87[0][0]             \n",
      "                                                                 dropout_88[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_106 (Dense)               (None, 24)           2112        concatenate_88[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_89 (Dropout)            (None, 24)           0           dense_106[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_89 (Concatenate)    (None, 111)          0           concatenate_88[0][0]             \n",
      "                                                                 dropout_89[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_107 (Dense)               (None, 24)           2688        concatenate_89[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_90 (Dropout)            (None, 24)           0           dense_107[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_90 (Concatenate)    (None, 135)          0           concatenate_89[0][0]             \n",
      "                                                                 dropout_90[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "highway_18 (Highway)            (None, 135)          36720       concatenate_90[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dense_108 (Dense)               (None, 3)            408         highway_18[0][0]                 \n",
      "==================================================================================================\n",
      "Total params: 44,808\n",
      "Trainable params: 44,808\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Train on 288497 samples, validate on 32055 samples\n",
      "Epoch 1/500\n",
      " - 3s - loss: 0.0223 - acc: 0.8756 - weighted_accuracy: 0.8541 - val_loss: 0.2895 - val_acc: 0.8703 - val_weighted_accuracy: 0.8667\n",
      "Epoch 2/500\n",
      " - 1s - loss: 0.0202 - acc: 0.8782 - weighted_accuracy: 0.8689 - val_loss: 0.2843 - val_acc: 0.8692 - val_weighted_accuracy: 0.8649\n",
      "Epoch 3/500\n",
      " - 2s - loss: 0.0200 - acc: 0.8782 - weighted_accuracy: 0.8689 - val_loss: 0.2871 - val_acc: 0.8725 - val_weighted_accuracy: 0.8666\n",
      "Epoch 4/500\n",
      " - 2s - loss: 0.0199 - acc: 0.8784 - weighted_accuracy: 0.8690 - val_loss: 0.2826 - val_acc: 0.8717 - val_weighted_accuracy: 0.8659\n",
      "Epoch 5/500\n",
      " - 2s - loss: 0.0199 - acc: 0.8781 - weighted_accuracy: 0.8691 - val_loss: 0.2817 - val_acc: 0.8723 - val_weighted_accuracy: 0.8670\n",
      "Epoch 6/500\n",
      " - 2s - loss: 0.0199 - acc: 0.8783 - weighted_accuracy: 0.8691 - val_loss: 0.2826 - val_acc: 0.8712 - val_weighted_accuracy: 0.8660\n",
      "Epoch 7/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8782 - weighted_accuracy: 0.8691 - val_loss: 0.2852 - val_acc: 0.8707 - val_weighted_accuracy: 0.8664\n",
      "Epoch 8/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8783 - weighted_accuracy: 0.8691 - val_loss: 0.2825 - val_acc: 0.8714 - val_weighted_accuracy: 0.8667\n",
      "Epoch 9/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8784 - weighted_accuracy: 0.8692 - val_loss: 0.2829 - val_acc: 0.8719 - val_weighted_accuracy: 0.8664\n",
      "Epoch 10/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8784 - weighted_accuracy: 0.8692 - val_loss: 0.2804 - val_acc: 0.8717 - val_weighted_accuracy: 0.8664\n",
      "Epoch 11/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8787 - weighted_accuracy: 0.8694 - val_loss: 0.2854 - val_acc: 0.8707 - val_weighted_accuracy: 0.8664\n",
      "Epoch 12/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8784 - weighted_accuracy: 0.8691 - val_loss: 0.2820 - val_acc: 0.8711 - val_weighted_accuracy: 0.8663\n",
      "Epoch 13/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8785 - weighted_accuracy: 0.8692 - val_loss: 0.2872 - val_acc: 0.8714 - val_weighted_accuracy: 0.8669\n",
      "Epoch 14/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8785 - weighted_accuracy: 0.8693 - val_loss: 0.2801 - val_acc: 0.8718 - val_weighted_accuracy: 0.8665\n",
      "Epoch 15/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8785 - weighted_accuracy: 0.8693 - val_loss: 0.2860 - val_acc: 0.8701 - val_weighted_accuracy: 0.8663\n",
      "Epoch 16/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8786 - weighted_accuracy: 0.8692 - val_loss: 0.2810 - val_acc: 0.8712 - val_weighted_accuracy: 0.8665\n",
      "Epoch 17/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8784 - weighted_accuracy: 0.8693 - val_loss: 0.2773 - val_acc: 0.8732 - val_weighted_accuracy: 0.8668\n",
      "Epoch 18/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8785 - weighted_accuracy: 0.8691 - val_loss: 0.2829 - val_acc: 0.8719 - val_weighted_accuracy: 0.8667\n",
      "Epoch 19/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8787 - weighted_accuracy: 0.8694 - val_loss: 0.2849 - val_acc: 0.8713 - val_weighted_accuracy: 0.8669\n",
      "Epoch 20/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8786 - weighted_accuracy: 0.8692 - val_loss: 0.2805 - val_acc: 0.8715 - val_weighted_accuracy: 0.8663\n",
      "Epoch 21/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8786 - weighted_accuracy: 0.8693 - val_loss: 0.2881 - val_acc: 0.8713 - val_weighted_accuracy: 0.8667\n",
      "Epoch 22/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8787 - weighted_accuracy: 0.8692 - val_loss: 0.2787 - val_acc: 0.8731 - val_weighted_accuracy: 0.8671\n",
      "Epoch 23/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8788 - weighted_accuracy: 0.8695 - val_loss: 0.2829 - val_acc: 0.8714 - val_weighted_accuracy: 0.8660\n",
      "Epoch 24/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8785 - weighted_accuracy: 0.8693 - val_loss: 0.2809 - val_acc: 0.8715 - val_weighted_accuracy: 0.8660\n",
      "Epoch 25/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8786 - weighted_accuracy: 0.8695 - val_loss: 0.2863 - val_acc: 0.8714 - val_weighted_accuracy: 0.8665\n",
      "Epoch 26/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8785 - weighted_accuracy: 0.8693 - val_loss: 0.2815 - val_acc: 0.8718 - val_weighted_accuracy: 0.8661\n",
      "Epoch 27/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8784 - weighted_accuracy: 0.8691 - val_loss: 0.2839 - val_acc: 0.8724 - val_weighted_accuracy: 0.8667\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "mata-features (InputLayer)      (None, 15)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_109 (Dense)               (None, 24)           384         mata-features[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_91 (Dropout)            (None, 24)           0           dense_109[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_91 (Concatenate)    (None, 39)           0           mata-features[0][0]              \n",
      "                                                                 dropout_91[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_110 (Dense)               (None, 24)           960         concatenate_91[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_92 (Dropout)            (None, 24)           0           dense_110[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_92 (Concatenate)    (None, 63)           0           concatenate_91[0][0]             \n",
      "                                                                 dropout_92[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_111 (Dense)               (None, 24)           1536        concatenate_92[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_93 (Dropout)            (None, 24)           0           dense_111[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_93 (Concatenate)    (None, 87)           0           concatenate_92[0][0]             \n",
      "                                                                 dropout_93[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_112 (Dense)               (None, 24)           2112        concatenate_93[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_94 (Dropout)            (None, 24)           0           dense_112[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_94 (Concatenate)    (None, 111)          0           concatenate_93[0][0]             \n",
      "                                                                 dropout_94[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_113 (Dense)               (None, 24)           2688        concatenate_94[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_95 (Dropout)            (None, 24)           0           dense_113[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_95 (Concatenate)    (None, 135)          0           concatenate_94[0][0]             \n",
      "                                                                 dropout_95[0][0]                 \n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "highway_19 (Highway)            (None, 135)          36720       concatenate_95[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dense_114 (Dense)               (None, 3)            408         highway_19[0][0]                 \n",
      "==================================================================================================\n",
      "Total params: 44,808\n",
      "Trainable params: 44,808\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Train on 288497 samples, validate on 32055 samples\n",
      "Epoch 1/500\n",
      " - 3s - loss: 0.0220 - acc: 0.8775 - weighted_accuracy: 0.8534 - val_loss: 0.3005 - val_acc: 0.8607 - val_weighted_accuracy: 0.8556\n",
      "Epoch 2/500\n",
      " - 1s - loss: 0.0199 - acc: 0.8806 - weighted_accuracy: 0.8706 - val_loss: 0.2995 - val_acc: 0.8645 - val_weighted_accuracy: 0.8541\n",
      "Epoch 3/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8805 - weighted_accuracy: 0.8706 - val_loss: 0.3012 - val_acc: 0.8632 - val_weighted_accuracy: 0.8552\n",
      "Epoch 4/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8804 - weighted_accuracy: 0.8707 - val_loss: 0.2990 - val_acc: 0.8652 - val_weighted_accuracy: 0.8544\n",
      "Epoch 5/500\n",
      " - 1s - loss: 0.0196 - acc: 0.8804 - weighted_accuracy: 0.8707 - val_loss: 0.2980 - val_acc: 0.8645 - val_weighted_accuracy: 0.8547\n",
      "Epoch 6/500\n",
      " - 1s - loss: 0.0196 - acc: 0.8805 - weighted_accuracy: 0.8709 - val_loss: 0.2992 - val_acc: 0.8643 - val_weighted_accuracy: 0.8544\n",
      "Epoch 7/500\n",
      " - 1s - loss: 0.0196 - acc: 0.8808 - weighted_accuracy: 0.8711 - val_loss: 0.2971 - val_acc: 0.8651 - val_weighted_accuracy: 0.8553\n",
      "Epoch 8/500\n",
      " - 1s - loss: 0.0196 - acc: 0.8801 - weighted_accuracy: 0.8706 - val_loss: 0.2942 - val_acc: 0.8661 - val_weighted_accuracy: 0.8536\n",
      "Epoch 9/500\n",
      " - 2s - loss: 0.0195 - acc: 0.8808 - weighted_accuracy: 0.8711 - val_loss: 0.2961 - val_acc: 0.8644 - val_weighted_accuracy: 0.8538\n",
      "Epoch 10/500\n",
      " - 1s - loss: 0.0196 - acc: 0.8803 - weighted_accuracy: 0.8709 - val_loss: 0.2979 - val_acc: 0.8643 - val_weighted_accuracy: 0.8537\n",
      "Epoch 11/500\n",
      " - 1s - loss: 0.0195 - acc: 0.8805 - weighted_accuracy: 0.8712 - val_loss: 0.2951 - val_acc: 0.8658 - val_weighted_accuracy: 0.8538\n",
      "Epoch 12/500\n",
      " - 1s - loss: 0.0195 - acc: 0.8806 - weighted_accuracy: 0.8711 - val_loss: 0.2910 - val_acc: 0.8666 - val_weighted_accuracy: 0.8526\n",
      "Epoch 13/500\n",
      " - 1s - loss: 0.0195 - acc: 0.8803 - weighted_accuracy: 0.8707 - val_loss: 0.2962 - val_acc: 0.8654 - val_weighted_accuracy: 0.8550\n",
      "Epoch 14/500\n",
      " - 1s - loss: 0.0195 - acc: 0.8807 - weighted_accuracy: 0.8711 - val_loss: 0.2939 - val_acc: 0.8688 - val_weighted_accuracy: 0.8534\n",
      "Epoch 15/500\n",
      " - 1s - loss: 0.0195 - acc: 0.8806 - weighted_accuracy: 0.8709 - val_loss: 0.2962 - val_acc: 0.8656 - val_weighted_accuracy: 0.8536\n",
      "Epoch 16/500\n",
      " - 1s - loss: 0.0195 - acc: 0.8805 - weighted_accuracy: 0.8710 - val_loss: 0.2909 - val_acc: 0.8680 - val_weighted_accuracy: 0.8536\n",
      "Epoch 17/500\n",
      " - 1s - loss: 0.0195 - acc: 0.8804 - weighted_accuracy: 0.8709 - val_loss: 0.2955 - val_acc: 0.8664 - val_weighted_accuracy: 0.8551\n",
      "Epoch 18/500\n",
      " - 1s - loss: 0.0195 - acc: 0.8808 - weighted_accuracy: 0.8712 - val_loss: 0.2944 - val_acc: 0.8662 - val_weighted_accuracy: 0.8551\n",
      "Epoch 19/500\n",
      " - 1s - loss: 0.0195 - acc: 0.8806 - weighted_accuracy: 0.8712 - val_loss: 0.2935 - val_acc: 0.8659 - val_weighted_accuracy: 0.8535\n",
      "Epoch 20/500\n",
      " - 1s - loss: 0.0195 - acc: 0.8807 - weighted_accuracy: 0.8710 - val_loss: 0.2944 - val_acc: 0.8658 - val_weighted_accuracy: 0.8548\n",
      "Epoch 21/500\n",
      " - 1s - loss: 0.0195 - acc: 0.8806 - weighted_accuracy: 0.8709 - val_loss: 0.2948 - val_acc: 0.8658 - val_weighted_accuracy: 0.8541\n",
      "Epoch 22/500\n",
      " - 1s - loss: 0.0195 - acc: 0.8806 - weighted_accuracy: 0.8711 - val_loss: 0.2938 - val_acc: 0.8663 - val_weighted_accuracy: 0.8526\n",
      "Epoch 23/500\n",
      " - 1s - loss: 0.0195 - acc: 0.8808 - weighted_accuracy: 0.8712 - val_loss: 0.2930 - val_acc: 0.8673 - val_weighted_accuracy: 0.8539\n",
      "Epoch 24/500\n",
      " - 1s - loss: 0.0195 - acc: 0.8809 - weighted_accuracy: 0.8711 - val_loss: 0.2937 - val_acc: 0.8662 - val_weighted_accuracy: 0.8537\n",
      "Epoch 25/500\n",
      " - 1s - loss: 0.0195 - acc: 0.8809 - weighted_accuracy: 0.8714 - val_loss: 0.2954 - val_acc: 0.8648 - val_weighted_accuracy: 0.8540\n",
      "Epoch 26/500\n",
      " - 1s - loss: 0.0195 - acc: 0.8807 - weighted_accuracy: 0.8711 - val_loss: 0.2978 - val_acc: 0.8658 - val_weighted_accuracy: 0.8547\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "mata-features (InputLayer)      (None, 15)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_115 (Dense)               (None, 24)           384         mata-features[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_96 (Dropout)            (None, 24)           0           dense_115[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_96 (Concatenate)    (None, 39)           0           mata-features[0][0]              \n",
      "                                                                 dropout_96[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_116 (Dense)               (None, 24)           960         concatenate_96[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_97 (Dropout)            (None, 24)           0           dense_116[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_97 (Concatenate)    (None, 63)           0           concatenate_96[0][0]             \n",
      "                                                                 dropout_97[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_117 (Dense)               (None, 24)           1536        concatenate_97[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_98 (Dropout)            (None, 24)           0           dense_117[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_98 (Concatenate)    (None, 87)           0           concatenate_97[0][0]             \n",
      "                                                                 dropout_98[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_118 (Dense)               (None, 24)           2112        concatenate_98[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_99 (Dropout)            (None, 24)           0           dense_118[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_99 (Concatenate)    (None, 111)          0           concatenate_98[0][0]             \n",
      "                                                                 dropout_99[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_119 (Dense)               (None, 24)           2688        concatenate_99[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_100 (Dropout)           (None, 24)           0           dense_119[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_100 (Concatenate)   (None, 135)          0           concatenate_99[0][0]             \n",
      "                                                                 dropout_100[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "highway_20 (Highway)            (None, 135)          36720       concatenate_100[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dense_120 (Dense)               (None, 3)            408         highway_20[0][0]                 \n",
      "==================================================================================================\n",
      "Total params: 44,808\n",
      "Trainable params: 44,808\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Train on 288497 samples, validate on 32055 samples\n",
      "Epoch 1/500\n",
      " - 3s - loss: 0.0236 - acc: 0.8659 - weighted_accuracy: 0.8434 - val_loss: 0.2803 - val_acc: 0.8731 - val_weighted_accuracy: 0.8649\n",
      "Epoch 2/500\n",
      " - 1s - loss: 0.0202 - acc: 0.8781 - weighted_accuracy: 0.8688 - val_loss: 0.2773 - val_acc: 0.8758 - val_weighted_accuracy: 0.8653\n",
      "Epoch 3/500\n",
      " - 1s - loss: 0.0201 - acc: 0.8785 - weighted_accuracy: 0.8694 - val_loss: 0.2834 - val_acc: 0.8730 - val_weighted_accuracy: 0.8647\n",
      "Epoch 4/500\n",
      " - 1s - loss: 0.0200 - acc: 0.8784 - weighted_accuracy: 0.8692 - val_loss: 0.2833 - val_acc: 0.8726 - val_weighted_accuracy: 0.8644\n",
      "Epoch 5/500\n",
      " - 1s - loss: 0.0199 - acc: 0.8784 - weighted_accuracy: 0.8694 - val_loss: 0.2837 - val_acc: 0.8709 - val_weighted_accuracy: 0.8644\n",
      "Epoch 6/500\n",
      " - 1s - loss: 0.0199 - acc: 0.8781 - weighted_accuracy: 0.8693 - val_loss: 0.2733 - val_acc: 0.8752 - val_weighted_accuracy: 0.8644\n",
      "Epoch 7/500\n",
      " - 1s - loss: 0.0199 - acc: 0.8785 - weighted_accuracy: 0.8694 - val_loss: 0.2751 - val_acc: 0.8745 - val_weighted_accuracy: 0.8650\n",
      "Epoch 8/500\n",
      " - 1s - loss: 0.0199 - acc: 0.8779 - weighted_accuracy: 0.8692 - val_loss: 0.2761 - val_acc: 0.8751 - val_weighted_accuracy: 0.8656\n",
      "Epoch 9/500\n",
      " - 1s - loss: 0.0199 - acc: 0.8783 - weighted_accuracy: 0.8697 - val_loss: 0.2760 - val_acc: 0.8744 - val_weighted_accuracy: 0.8642\n",
      "Epoch 10/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8783 - weighted_accuracy: 0.8699 - val_loss: 0.2798 - val_acc: 0.8726 - val_weighted_accuracy: 0.8645\n",
      "Epoch 11/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8784 - weighted_accuracy: 0.8694 - val_loss: 0.2760 - val_acc: 0.8742 - val_weighted_accuracy: 0.8649\n",
      "Epoch 12/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8782 - weighted_accuracy: 0.8695 - val_loss: 0.2751 - val_acc: 0.8750 - val_weighted_accuracy: 0.8651\n",
      "Epoch 13/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8785 - weighted_accuracy: 0.8696 - val_loss: 0.2732 - val_acc: 0.8752 - val_weighted_accuracy: 0.8643\n",
      "Epoch 14/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8783 - weighted_accuracy: 0.8697 - val_loss: 0.2751 - val_acc: 0.8749 - val_weighted_accuracy: 0.8642\n",
      "Epoch 15/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8786 - weighted_accuracy: 0.8694 - val_loss: 0.2724 - val_acc: 0.8758 - val_weighted_accuracy: 0.8645\n",
      "Epoch 16/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8786 - weighted_accuracy: 0.8699 - val_loss: 0.2825 - val_acc: 0.8726 - val_weighted_accuracy: 0.8654\n",
      "Epoch 17/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8783 - weighted_accuracy: 0.8695 - val_loss: 0.2745 - val_acc: 0.8748 - val_weighted_accuracy: 0.8644\n",
      "Epoch 18/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8782 - weighted_accuracy: 0.8692 - val_loss: 0.2817 - val_acc: 0.8737 - val_weighted_accuracy: 0.8654\n",
      "Epoch 19/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8786 - weighted_accuracy: 0.8698 - val_loss: 0.2730 - val_acc: 0.8766 - val_weighted_accuracy: 0.8648\n",
      "Epoch 20/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8784 - weighted_accuracy: 0.8694 - val_loss: 0.2786 - val_acc: 0.8737 - val_weighted_accuracy: 0.8650\n",
      "Epoch 21/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8785 - weighted_accuracy: 0.8696 - val_loss: 0.2830 - val_acc: 0.8706 - val_weighted_accuracy: 0.8644\n",
      "Epoch 22/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8785 - weighted_accuracy: 0.8696 - val_loss: 0.2714 - val_acc: 0.8755 - val_weighted_accuracy: 0.8644\n",
      "Epoch 23/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8786 - weighted_accuracy: 0.8697 - val_loss: 0.2755 - val_acc: 0.8743 - val_weighted_accuracy: 0.8652\n",
      "Epoch 24/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8785 - weighted_accuracy: 0.8695 - val_loss: 0.2784 - val_acc: 0.8731 - val_weighted_accuracy: 0.8649\n",
      "Epoch 25/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8784 - weighted_accuracy: 0.8696 - val_loss: 0.2758 - val_acc: 0.8734 - val_weighted_accuracy: 0.8646\n",
      "Epoch 26/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8785 - weighted_accuracy: 0.8696 - val_loss: 0.2809 - val_acc: 0.8699 - val_weighted_accuracy: 0.8637\n",
      "Epoch 27/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8782 - weighted_accuracy: 0.8697 - val_loss: 0.2753 - val_acc: 0.8752 - val_weighted_accuracy: 0.8651\n",
      "Epoch 28/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8787 - weighted_accuracy: 0.8697 - val_loss: 0.2764 - val_acc: 0.8738 - val_weighted_accuracy: 0.8651\n",
      "Epoch 29/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8783 - weighted_accuracy: 0.8697 - val_loss: 0.2768 - val_acc: 0.8741 - val_weighted_accuracy: 0.8652\n",
      "Epoch 30/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8785 - weighted_accuracy: 0.8696 - val_loss: 0.2751 - val_acc: 0.8753 - val_weighted_accuracy: 0.8658\n",
      "Epoch 31/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8785 - weighted_accuracy: 0.8696 - val_loss: 0.2768 - val_acc: 0.8733 - val_weighted_accuracy: 0.8645\n",
      "Epoch 32/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8785 - weighted_accuracy: 0.8698 - val_loss: 0.2703 - val_acc: 0.8765 - val_weighted_accuracy: 0.8647\n",
      "Epoch 33/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8786 - weighted_accuracy: 0.8697 - val_loss: 0.2753 - val_acc: 0.8751 - val_weighted_accuracy: 0.8654\n",
      "Epoch 34/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8788 - weighted_accuracy: 0.8698 - val_loss: 0.2778 - val_acc: 0.8738 - val_weighted_accuracy: 0.8652\n",
      "Epoch 35/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8787 - weighted_accuracy: 0.8697 - val_loss: 0.2763 - val_acc: 0.8732 - val_weighted_accuracy: 0.8644\n",
      "Epoch 36/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8786 - weighted_accuracy: 0.8697 - val_loss: 0.2737 - val_acc: 0.8747 - val_weighted_accuracy: 0.8647\n",
      "Epoch 37/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8789 - weighted_accuracy: 0.8699 - val_loss: 0.2798 - val_acc: 0.8713 - val_weighted_accuracy: 0.8645\n",
      "Epoch 38/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8784 - weighted_accuracy: 0.8697 - val_loss: 0.2755 - val_acc: 0.8743 - val_weighted_accuracy: 0.8649\n",
      "Epoch 39/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8783 - weighted_accuracy: 0.8696 - val_loss: 0.2748 - val_acc: 0.8761 - val_weighted_accuracy: 0.8653\n",
      "Epoch 40/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8785 - weighted_accuracy: 0.8695 - val_loss: 0.2765 - val_acc: 0.8728 - val_weighted_accuracy: 0.8650\n",
      "Epoch 41/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8784 - weighted_accuracy: 0.8698 - val_loss: 0.2754 - val_acc: 0.8745 - val_weighted_accuracy: 0.8642\n",
      "Epoch 42/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8788 - weighted_accuracy: 0.8700 - val_loss: 0.2750 - val_acc: 0.8747 - val_weighted_accuracy: 0.8643\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "mata-features (InputLayer)      (None, 15)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_121 (Dense)               (None, 24)           384         mata-features[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_101 (Dropout)           (None, 24)           0           dense_121[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_101 (Concatenate)   (None, 39)           0           mata-features[0][0]              \n",
      "                                                                 dropout_101[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_122 (Dense)               (None, 24)           960         concatenate_101[0][0]            \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "dropout_102 (Dropout)           (None, 24)           0           dense_122[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_102 (Concatenate)   (None, 63)           0           concatenate_101[0][0]            \n",
      "                                                                 dropout_102[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_123 (Dense)               (None, 24)           1536        concatenate_102[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_103 (Dropout)           (None, 24)           0           dense_123[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_103 (Concatenate)   (None, 87)           0           concatenate_102[0][0]            \n",
      "                                                                 dropout_103[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_124 (Dense)               (None, 24)           2112        concatenate_103[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_104 (Dropout)           (None, 24)           0           dense_124[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_104 (Concatenate)   (None, 111)          0           concatenate_103[0][0]            \n",
      "                                                                 dropout_104[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_125 (Dense)               (None, 24)           2688        concatenate_104[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_105 (Dropout)           (None, 24)           0           dense_125[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_105 (Concatenate)   (None, 135)          0           concatenate_104[0][0]            \n",
      "                                                                 dropout_105[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "highway_21 (Highway)            (None, 135)          36720       concatenate_105[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dense_126 (Dense)               (None, 3)            408         highway_21[0][0]                 \n",
      "==================================================================================================\n",
      "Total params: 44,808\n",
      "Trainable params: 44,808\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Train on 288497 samples, validate on 32055 samples\n",
      "Epoch 1/500\n",
      " - 3s - loss: 0.0223 - acc: 0.8745 - weighted_accuracy: 0.8563 - val_loss: 0.2882 - val_acc: 0.8763 - val_weighted_accuracy: 0.8727\n",
      "Epoch 2/500\n",
      " - 1s - loss: 0.0203 - acc: 0.8774 - weighted_accuracy: 0.8680 - val_loss: 0.2687 - val_acc: 0.8835 - val_weighted_accuracy: 0.8751\n",
      "Epoch 3/500\n",
      " - 1s - loss: 0.0201 - acc: 0.8779 - weighted_accuracy: 0.8684 - val_loss: 0.2738 - val_acc: 0.8802 - val_weighted_accuracy: 0.8745\n",
      "Epoch 4/500\n",
      " - 1s - loss: 0.0200 - acc: 0.8777 - weighted_accuracy: 0.8686 - val_loss: 0.2682 - val_acc: 0.8829 - val_weighted_accuracy: 0.8755\n",
      "Epoch 5/500\n",
      " - 1s - loss: 0.0200 - acc: 0.8776 - weighted_accuracy: 0.8686 - val_loss: 0.2708 - val_acc: 0.8828 - val_weighted_accuracy: 0.8753\n",
      "Epoch 6/500\n",
      " - 1s - loss: 0.0199 - acc: 0.8775 - weighted_accuracy: 0.8683 - val_loss: 0.2702 - val_acc: 0.8815 - val_weighted_accuracy: 0.8748\n",
      "Epoch 7/500\n",
      " - 1s - loss: 0.0199 - acc: 0.8774 - weighted_accuracy: 0.8684 - val_loss: 0.2731 - val_acc: 0.8796 - val_weighted_accuracy: 0.8742\n",
      "Epoch 8/500\n",
      " - 1s - loss: 0.0199 - acc: 0.8778 - weighted_accuracy: 0.8687 - val_loss: 0.2701 - val_acc: 0.8830 - val_weighted_accuracy: 0.8758\n",
      "Epoch 9/500\n",
      " - 1s - loss: 0.0199 - acc: 0.8778 - weighted_accuracy: 0.8685 - val_loss: 0.2723 - val_acc: 0.8802 - val_weighted_accuracy: 0.8742\n",
      "Epoch 10/500\n",
      " - 1s - loss: 0.0199 - acc: 0.8775 - weighted_accuracy: 0.8684 - val_loss: 0.2781 - val_acc: 0.8796 - val_weighted_accuracy: 0.8742\n",
      "Epoch 11/500\n",
      " - 1s - loss: 0.0199 - acc: 0.8777 - weighted_accuracy: 0.8685 - val_loss: 0.2724 - val_acc: 0.8803 - val_weighted_accuracy: 0.8744\n",
      "Epoch 12/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8780 - weighted_accuracy: 0.8686 - val_loss: 0.2723 - val_acc: 0.8798 - val_weighted_accuracy: 0.8743\n",
      "Epoch 13/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8780 - weighted_accuracy: 0.8687 - val_loss: 0.2716 - val_acc: 0.8803 - val_weighted_accuracy: 0.8741\n",
      "Epoch 14/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8774 - weighted_accuracy: 0.8687 - val_loss: 0.2699 - val_acc: 0.8820 - val_weighted_accuracy: 0.8751\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "mata-features (InputLayer)      (None, 15)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_127 (Dense)               (None, 24)           384         mata-features[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_106 (Dropout)           (None, 24)           0           dense_127[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_106 (Concatenate)   (None, 39)           0           mata-features[0][0]              \n",
      "                                                                 dropout_106[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_128 (Dense)               (None, 24)           960         concatenate_106[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_107 (Dropout)           (None, 24)           0           dense_128[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_107 (Concatenate)   (None, 63)           0           concatenate_106[0][0]            \n",
      "                                                                 dropout_107[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_129 (Dense)               (None, 24)           1536        concatenate_107[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_108 (Dropout)           (None, 24)           0           dense_129[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_108 (Concatenate)   (None, 87)           0           concatenate_107[0][0]            \n",
      "                                                                 dropout_108[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_130 (Dense)               (None, 24)           2112        concatenate_108[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_109 (Dropout)           (None, 24)           0           dense_130[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_109 (Concatenate)   (None, 111)          0           concatenate_108[0][0]            \n",
      "                                                                 dropout_109[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_131 (Dense)               (None, 24)           2688        concatenate_109[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_110 (Dropout)           (None, 24)           0           dense_131[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_110 (Concatenate)   (None, 135)          0           concatenate_109[0][0]            \n",
      "                                                                 dropout_110[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "highway_22 (Highway)            (None, 135)          36720       concatenate_110[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dense_132 (Dense)               (None, 3)            408         highway_22[0][0]                 \n",
      "==================================================================================================\n",
      "Total params: 44,808\n",
      "Trainable params: 44,808\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Train on 288495 samples, validate on 32057 samples\n",
      "Epoch 1/500\n",
      " - 3s - loss: 0.0230 - acc: 0.8704 - weighted_accuracy: 0.8486 - val_loss: 0.2626 - val_acc: 0.8850 - val_weighted_accuracy: 0.8707\n",
      "Epoch 2/500\n",
      " - 1s - loss: 0.0203 - acc: 0.8774 - weighted_accuracy: 0.8685 - val_loss: 0.2654 - val_acc: 0.8825 - val_weighted_accuracy: 0.8716\n",
      "Epoch 3/500\n",
      " - 1s - loss: 0.0201 - acc: 0.8775 - weighted_accuracy: 0.8686 - val_loss: 0.2654 - val_acc: 0.8830 - val_weighted_accuracy: 0.8712\n",
      "Epoch 4/500\n",
      " - 1s - loss: 0.0200 - acc: 0.8774 - weighted_accuracy: 0.8687 - val_loss: 0.2774 - val_acc: 0.8797 - val_weighted_accuracy: 0.8704\n",
      "Epoch 5/500\n",
      " - 1s - loss: 0.0200 - acc: 0.8774 - weighted_accuracy: 0.8688 - val_loss: 0.2693 - val_acc: 0.8805 - val_weighted_accuracy: 0.8701\n",
      "Epoch 6/500\n",
      " - 1s - loss: 0.0199 - acc: 0.8776 - weighted_accuracy: 0.8691 - val_loss: 0.2642 - val_acc: 0.8815 - val_weighted_accuracy: 0.8705\n",
      "Epoch 7/500\n",
      " - 1s - loss: 0.0199 - acc: 0.8771 - weighted_accuracy: 0.8688 - val_loss: 0.2617 - val_acc: 0.8825 - val_weighted_accuracy: 0.8706\n",
      "Epoch 8/500\n",
      " - 1s - loss: 0.0199 - acc: 0.8772 - weighted_accuracy: 0.8689 - val_loss: 0.2576 - val_acc: 0.8850 - val_weighted_accuracy: 0.8704\n",
      "Epoch 9/500\n",
      " - 1s - loss: 0.0199 - acc: 0.8775 - weighted_accuracy: 0.8689 - val_loss: 0.2627 - val_acc: 0.8819 - val_weighted_accuracy: 0.8711\n",
      "Epoch 10/500\n",
      " - 1s - loss: 0.0199 - acc: 0.8771 - weighted_accuracy: 0.8686 - val_loss: 0.2598 - val_acc: 0.8845 - val_weighted_accuracy: 0.8704\n",
      "Epoch 11/500\n",
      " - 1s - loss: 0.0199 - acc: 0.8776 - weighted_accuracy: 0.8688 - val_loss: 0.2651 - val_acc: 0.8830 - val_weighted_accuracy: 0.8707\n",
      "Epoch 12/500\n",
      " - 1s - loss: 0.0199 - acc: 0.8776 - weighted_accuracy: 0.8689 - val_loss: 0.2603 - val_acc: 0.8829 - val_weighted_accuracy: 0.8703\n",
      "Epoch 13/500\n",
      " - 1s - loss: 0.0199 - acc: 0.8773 - weighted_accuracy: 0.8690 - val_loss: 0.2671 - val_acc: 0.8826 - val_weighted_accuracy: 0.8711\n",
      "Epoch 14/500\n",
      " - 1s - loss: 0.0199 - acc: 0.8774 - weighted_accuracy: 0.8691 - val_loss: 0.2612 - val_acc: 0.8821 - val_weighted_accuracy: 0.8708\n",
      "Epoch 15/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8774 - weighted_accuracy: 0.8689 - val_loss: 0.2642 - val_acc: 0.8819 - val_weighted_accuracy: 0.8706\n",
      "Epoch 16/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8775 - weighted_accuracy: 0.8690 - val_loss: 0.2590 - val_acc: 0.8844 - val_weighted_accuracy: 0.8705\n",
      "Epoch 17/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8775 - weighted_accuracy: 0.8690 - val_loss: 0.2611 - val_acc: 0.8826 - val_weighted_accuracy: 0.8711\n",
      "Epoch 18/500\n",
      " - 1s - loss: 0.0198 - acc: 0.8774 - weighted_accuracy: 0.8691 - val_loss: 0.2621 - val_acc: 0.8826 - val_weighted_accuracy: 0.8708\n"
     ]
    }
   ],
   "source": [
    "trainer = KerasModelTrainer(model_stamp=\"Ensemble-DenseNet\", epoch_num=500)\n",
    "models, score, folds_preds = trainer.train_folds(features=ensemble_trains, y=to_categorical(labels), augments=None, fold_count=10,\n",
    "    batch_size=1024, \n",
    "    scale_sample_weight=None, class_weight=None,\n",
    "    get_model_func=_agent_get_model, \n",
    "    patience=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8689630966648629"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8689630966648629"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "score 0.8689630966648629\n",
      "Predicting training results...\n",
      "Predicting testing results...\n",
      "80126/80126 [==============================] - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - 1s 8us/step\n",
      "80126/80126 [==============================] - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - 1s 8us/step\n",
      "80126/80126 [==============================] - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - 1s 8us/step\n",
      "80126/80126 [==============================] - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - 1s 8us/step\n",
      "80126/80126 [==============================] - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - 1s 8us/step\n",
      "80126/80126 [==============================] - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - 1s 9us/step\n",
      "80126/80126 [==============================] - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - 1s 8us/step\n",
      "80126/80126 [==============================] - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - 1s 8us/step\n",
      "80126/80126 [==============================] - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - 1s 9us/step\n",
      "80126/80126 [==============================] - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - 1s 9us/step\n",
      "Predicting labeled testing results...\n"
     ]
    }
   ],
   "source": [
    "print(\"score\", score)\n",
    "oofs_dir = \"../data/ensemble/second_level/oofs/\"\n",
    "output_dir = \"../data/ensemble/second_level/preds/\"\n",
    "onehot_pred_dir = \"../data/ensemble/second_level/nn_one_hot/\"\n",
    "\n",
    "model_submit_prefix = \"AddNN-Ensemble\"\n",
    "\n",
    "oofs_path = oofs_dir + model_submit_prefix\n",
    "output_path = output_dir + model_submit_prefix\n",
    "one_hot_pred_path = onehot_pred_dir + \"One-Hot\" + model_submit_prefix\n",
    "\n",
    "print(\"Predicting training results...\")\n",
    "train_predicts = np.concatenate(folds_preds, axis=0)\n",
    "score = np_weighted_accuracy(to_categorical(labels), train_predicts)\n",
    "\n",
    "oofs = pd.DataFrame({\"unrelated\": train_predicts[:, 0], \"agreed\": train_predicts[:, 1], \"disagreed\": train_predicts[:, 2]})\n",
    "submit_path = oofs_path + \"-Train-L{:4f}-NB{:d}.csv\".format(score, NB_WORDS)\n",
    "oofs.to_csv(submit_path, index=False)\n",
    "\n",
    "print(\"Predicting testing results...\")\n",
    "test_predicts_list = []\n",
    "for fold_id, model in enumerate(models):\n",
    "    test_predicts = model.predict({\"mata-features\": ensemble_tests}, batch_size=128, verbose=1)\n",
    "    test_predicts_list.append(test_predicts)\n",
    "\n",
    "test_predicts = np.zeros(test_predicts_list[0].shape)\n",
    "for fold_predict in test_predicts_list:\n",
    "    test_predicts += fold_predict\n",
    "test_predicts /= len(test_predicts_list)\n",
    "\n",
    "test_predicts = pd.DataFrame({\"unrelated\": test_predicts[:, 0], \"agreed\": test_predicts[:, 1], \"disagreed\": test_predicts[:, 2]})\n",
    "submit_path = output_path + \"-L{:4f}-NB{:d}.csv\".format(score, NB_WORDS)\n",
    "test_predicts.to_csv(submit_path, index=False) # 0.3343\n",
    "\n",
    "print(\"Predicting labeled testing results...\")\n",
    "ids = pd.read_csv(\"../data/dataset/test.csv\")\n",
    "pred_labels = test_predicts.idxmax(axis=1)\n",
    "sub = pd.DataFrame({\"Id\": ids['id'].values, \"Category\": pred_labels})\n",
    "submit_path = one_hot_pred_path + \"-L{:4f}-NB{:d}.csv\".format(score, NB_WORDS)\n",
    "sub.to_csv(submit_path, index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 1,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
